{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b85f2de6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "/usr/local/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:526: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/usr/local/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:527: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/usr/local/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:528: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/usr/local/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:529: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/usr/local/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:530: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/usr/local/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:535: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from models.MTL_IBA import MTL_IBA, MTL_IBA_h3, MTL_IBA_cross, MTL_IBA_cross2, MTL_IBA_cross3\n",
    "from models.MLT_net import MTL_classic\n",
    "from keras.applications.vgg16 import preprocess_input, decode_predictions\n",
    "from keras.models import load_model, Model\n",
    "from keras.preprocessing import image\n",
    "import matplotlib.pyplot as plt\n",
    "from keras import backend as K\n",
    "import numpy as np\n",
    "import cv2\n",
    "\n",
    "\n",
    "\n",
    "import os\n",
    "\n",
    "\n",
    "# weight_path = 'MTL_IBA_cross3_256.h5'\n",
    "\n",
    "# base_model = MTL_IBA_cross3(256, 256, 3, nClasses=2)\n",
    "# # model = MTL_IBA_cross(img_size, img_size, depth, nClasses=2)\n",
    "# # model = MTL_IBA_cross2(img_size, img_size, depth, nClasses=2)\n",
    "# # model = MTL_IBA_cross3(img_size, img_size, depth, nClasses=2)\n",
    "\n",
    "# base_model.load_weights(weight_path, by_name=True)\n",
    "# base_model.summary()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "065742ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_img_preprocess(img_path, target_size):\n",
    "    img = image.load_img(img_path, target_size=target_size)\n",
    "    img = image.img_to_array(img) \n",
    "    img = np.expand_dims(img, axis=0) # \n",
    "    img = preprocess_input(img) #\n",
    "    return img\n",
    "\n",
    "\n",
    "def gradient_compute(model, img):\n",
    "  \n",
    "    preds = model.predict(img)\n",
    "    num = np.argmax(preds[0]) #\n",
    "    \n",
    "    model2 = Model(inputs=model.input, outputs=model.get_layer('block5_c_conv3').output)\n",
    "\n",
    "    output = model2.output[0][:, num]\n",
    "    last_layer = model2.get_layer('block5_c_conv3')\n",
    "    grads = K.gradients(output, last_layer.output)[0]\n",
    "    pooled_grads = K.mean(grads, axis=(0, 1, 2)) #\n",
    "    iterate = K.function([model2.input], [pooled_grads, last_layer.output[0]])\n",
    "    pooled_grads_value, conv_layer_output_value = iterate([img])\n",
    "\n",
    "    for i in range(pooled_grads.shape[0]):\n",
    "        conv_layer_output_value[:, :, i] *= pooled_grads_value[i]\n",
    "\n",
    "    return conv_layer_output_value\n",
    "\n",
    "def plot_heatmap(conv_layer_output_value, img_in_path, img_out_path):\n",
    "    \n",
    "\n",
    "    mean = np.mean(conv_layer_output_value, axis=0)\n",
    "    maxi = np.max(conv_layer_output_value)\n",
    "    mini = np.min(conv_layer_output_value)\n",
    "#     conv_layer_output_value = (conv_layer_output_value - mean)/(maxi-mini)\n",
    "#     conv_layer_output_value = (conv_layer_output_value - mean)\n",
    "    heatmap = np.mean(conv_layer_output_value,axis = -1)\n",
    "    heatmap = np.maximum(heatmap, 0)\n",
    "    mean = np.mean(heatmap, axis =0)\n",
    "    heatmap = heatmap / np.max(heatmap)\n",
    "\n",
    "    img = cv2.imread(img_in_path)\n",
    "    heatmap = cv2.resize(heatmap, (img.shape[1], img.shape[0]))\n",
    "    heatmap = np.uint8(255 * heatmap)\n",
    "\n",
    "    heatmap = cv2.applyColorMap(heatmap, cv2.COLORMAP_JET)\n",
    "    superimopsed_img = heatmap * 0.5 + img\n",
    "    \n",
    "    cv2.imwrite(img_out_path, superimopsed_img)\n",
    "    \n",
    "    print(img_out_path)\n",
    "\n",
    "    \n",
    "def plot_dict_img_heatmap(testX_dir, save_dir, model):\n",
    "    \n",
    "    class_list = os.listdir(testX_dir)\n",
    "    for class_name in class_list:\n",
    "        class_path = os.path.join(testX_dir, class_name)\n",
    "        img_list = os.listdir(class_path)\n",
    "        for image_Name in img_list:\n",
    "            \n",
    "            img_path = os.path.join(class_path, image_Name)\n",
    "            print(img_path)\n",
    "            img_out_path = os.path.join(save_dir, image_Name)\n",
    "            \n",
    "            img = load_img_preprocess(img_path, (img_size, img_size))\n",
    "            conv_value = gradient_compute(model, img)\n",
    "            plot_heatmap(conv_value, img_path, img_out_path)\n",
    "            \n",
    "\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "63e4365b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dataset/Dataset_BUSI_AN/test/images/bengin_images/benign (51).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/benign (51).png\n",
      "dataset/Dataset_BUSI_AN/test/images/bengin_images/benign (299).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/benign (299).png\n",
      "dataset/Dataset_BUSI_AN/test/images/bengin_images/benign (167).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/benign (167).png\n",
      "dataset/Dataset_BUSI_AN/test/images/bengin_images/benign (220).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/benign (220).png\n",
      "dataset/Dataset_BUSI_AN/test/images/bengin_images/benign (297).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/benign (297).png\n",
      "dataset/Dataset_BUSI_AN/test/images/bengin_images/benign (425).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/benign (425).png\n",
      "dataset/Dataset_BUSI_AN/test/images/bengin_images/benign (57).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/benign (57).png\n",
      "dataset/Dataset_BUSI_AN/test/images/bengin_images/benign (38).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/benign (38).png\n",
      "dataset/Dataset_BUSI_AN/test/images/bengin_images/benign (222).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/benign (222).png\n",
      "dataset/Dataset_BUSI_AN/test/images/bengin_images/benign (372).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/benign (372).png\n",
      "dataset/Dataset_BUSI_AN/test/images/bengin_images/benign (140).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/benign (140).png\n",
      "dataset/Dataset_BUSI_AN/test/images/bengin_images/benign (155).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/benign (155).png\n",
      "dataset/Dataset_BUSI_AN/test/images/bengin_images/benign (321).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/benign (321).png\n",
      "dataset/Dataset_BUSI_AN/test/images/bengin_images/benign (15).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/benign (15).png\n",
      "dataset/Dataset_BUSI_AN/test/images/bengin_images/benign (103).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/benign (103).png\n",
      "dataset/Dataset_BUSI_AN/test/images/bengin_images/benign (154).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/benign (154).png\n",
      "dataset/Dataset_BUSI_AN/test/images/bengin_images/benign (221).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/benign (221).png\n",
      "dataset/Dataset_BUSI_AN/test/images/bengin_images/benign (136).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/benign (136).png\n",
      "dataset/Dataset_BUSI_AN/test/images/bengin_images/benign (187).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/benign (187).png\n",
      "dataset/Dataset_BUSI_AN/test/images/bengin_images/benign (218).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/benign (218).png\n",
      "dataset/Dataset_BUSI_AN/test/images/bengin_images/benign (231).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/benign (231).png\n",
      "dataset/Dataset_BUSI_AN/test/images/bengin_images/benign (127).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/benign (127).png\n",
      "dataset/Dataset_BUSI_AN/test/images/bengin_images/benign (21).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/benign (21).png\n",
      "dataset/Dataset_BUSI_AN/test/images/bengin_images/benign (150).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/benign (150).png\n",
      "dataset/Dataset_BUSI_AN/test/images/bengin_images/benign (315).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/benign (315).png\n",
      "dataset/Dataset_BUSI_AN/test/images/bengin_images/benign (104).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/benign (104).png\n",
      "dataset/Dataset_BUSI_AN/test/images/bengin_images/benign (142).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/benign (142).png\n",
      "dataset/Dataset_BUSI_AN/test/images/bengin_images/benign (119).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/benign (119).png\n",
      "dataset/Dataset_BUSI_AN/test/images/bengin_images/benign (153).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/benign (153).png\n",
      "dataset/Dataset_BUSI_AN/test/images/bengin_images/benign (88).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/benign (88).png\n",
      "dataset/Dataset_BUSI_AN/test/images/bengin_images/benign (388).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/benign (388).png\n",
      "dataset/Dataset_BUSI_AN/test/images/bengin_images/benign (48).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/benign (48).png\n",
      "dataset/Dataset_BUSI_AN/test/images/bengin_images/benign (316).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/benign (316).png\n",
      "dataset/Dataset_BUSI_AN/test/images/bengin_images/benign (81).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/benign (81).png\n",
      "dataset/Dataset_BUSI_AN/test/images/bengin_images/benign (349).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/benign (349).png\n",
      "dataset/Dataset_BUSI_AN/test/images/bengin_images/benign (84).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/benign (84).png\n",
      "dataset/Dataset_BUSI_AN/test/images/bengin_images/benign (364).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/benign (364).png\n",
      "dataset/Dataset_BUSI_AN/test/images/bengin_images/benign (353).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/benign (353).png\n",
      "dataset/Dataset_BUSI_AN/test/images/bengin_images/benign (408).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/benign (408).png\n",
      "dataset/Dataset_BUSI_AN/test/images/bengin_images/benign (324).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/benign (324).png\n",
      "dataset/Dataset_BUSI_AN/test/images/bengin_images/benign (391).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/benign (391).png\n",
      "dataset/Dataset_BUSI_AN/test/images/bengin_images/benign (147).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/benign (147).png\n",
      "dataset/Dataset_BUSI_AN/test/images/bengin_images/benign (201).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/benign (201).png\n",
      "dataset/Dataset_BUSI_AN/test/images/malignant_images/malignant (191).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/malignant (191).png\n",
      "dataset/Dataset_BUSI_AN/test/images/malignant_images/malignant (170).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/malignant (170).png\n",
      "dataset/Dataset_BUSI_AN/test/images/malignant_images/malignant (131).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/malignant (131).png\n",
      "dataset/Dataset_BUSI_AN/test/images/malignant_images/malignant (112).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/malignant (112).png\n",
      "dataset/Dataset_BUSI_AN/test/images/malignant_images/malignant (190).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/malignant (190).png\n",
      "dataset/Dataset_BUSI_AN/test/images/malignant_images/malignant (18).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/malignant (18).png\n",
      "dataset/Dataset_BUSI_AN/test/images/malignant_images/malignant (44).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/malignant (44).png\n",
      "dataset/Dataset_BUSI_AN/test/images/malignant_images/malignant (200).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/malignant (200).png\n",
      "dataset/Dataset_BUSI_AN/test/images/malignant_images/malignant (149).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/malignant (149).png\n",
      "dataset/Dataset_BUSI_AN/test/images/malignant_images/malignant (86).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/malignant (86).png\n",
      "dataset/Dataset_BUSI_AN/test/images/malignant_images/malignant (123).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/malignant (123).png\n",
      "dataset/Dataset_BUSI_AN/test/images/malignant_images/malignant (20).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/malignant (20).png\n",
      "dataset/Dataset_BUSI_AN/test/images/malignant_images/malignant (74).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/malignant (74).png\n",
      "dataset/Dataset_BUSI_AN/test/images/malignant_images/malignant (121).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/malignant (121).png\n",
      "dataset/Dataset_BUSI_AN/test/images/malignant_images/malignant (146).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/malignant (146).png\n",
      "dataset/Dataset_BUSI_AN/test/images/malignant_images/malignant (72).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/malignant (72).png\n",
      "dataset/Dataset_BUSI_AN/test/images/malignant_images/malignant (171).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/malignant (171).png\n",
      "dataset/Dataset_BUSI_AN/test/images/malignant_images/malignant (106).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/malignant (106).png\n",
      "dataset/Dataset_BUSI_AN/test/images/malignant_images/malignant (45).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/malignant (45).png\n",
      "dataset/Dataset_BUSI_AN/test/images/malignant_images/malignant (65).png\n",
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/malignant (65).png\n",
      "dataset/Dataset_BUSI_AN/test/images/malignant_images/malignant (136).png\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dataset/prediction/Dataset_BUSI_AN/MTL_GCT2/malignant (136).png\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "from models.MTL_IBA import MTL_IBA, MTL_IBA_h3, MTL_IBA_cross, MTL_IBA_cross2, MTL_IBA_cross3\n",
    "from models.MLT_net import MTL_classic\n",
    "from models.MTL_Attention import MTL_Attention_model\n",
    "\n",
    "Name =  'MTL_GCT2'\n",
    "img_size =224\n",
    "depth = 3\n",
    "testX_dir = 'dataset/Dataset_BUSI_AN/test/images/'\n",
    "save_dir = 'dataset/prediction/Dataset_BUSI_AN/' + Name\n",
    "if not os.path.exists(save_dir):\n",
    "    os.makedirs(save_dir)\n",
    "    \n",
    "    \n",
    "\n",
    "os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\"\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n",
    "   \n",
    "# model = MTL_IBA_cross3(img_size, img_size, depth, nClasses=2)\n",
    "model = MTL_Attention_model(img_size, img_size, depth, nClasses=2)\n",
    "model.load_weights('MTL_GCT2.h5', by_name=True)\n",
    "\n",
    "\n",
    "\n",
    "plot_dict_img_heatmap(testX_dir, save_dir, model)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c4144c5a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
